package page;

import java.io.IOException;
import java.util.Collection;
import java.util.HashSet;
import java.util.NoSuchElementException;
import java.util.Set;
import java.util.concurrent.LinkedBlockingQueue;

import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;

/**
 * PageRetriever class retrieves web pages and stores
 * them for later analysis by the PageParser class.
 * @author Vladimir Dudnic, Derek Fleener, Michael Groenier
 */
public class PageRetriever
{
        /** 
         * Queue of web pages waiting to be parsed. 
         */
        LinkedBlockingQueue<String> waiting;

        /** 
         * Set of URLs already visited.
         */
        Set<String> visited;
       
        /**
         * Constructor that initializes with a starting address.
         *
         * @param starting_address The first address.
         * @throws IOException if not acceptable address.
         */
        public PageRetriever(String starting_address) throws IOException 
        {
                waiting = new LinkedBlockingQueue<String>();
                visited = new HashSet<String>();
               
                waiting.add(starting_address);
        }
       
        /**
         * Adds a collection of unvisited web pages.
         *
         * @param addresses The collection of addresses.
         */
        public void addAllPages(Collection<? extends String> addresses) 
        {
                for (String address : addresses) 
                {
                        waiting.add(address);
                }
        }
       
        /**
         * @return the next available web page's content.
         * @throws NoSuchElementException if queue is empty
         */
        public Document getNextPage()
        {
                Document doc = null;
                boolean validURL = false;
                String next = waiting.remove();
                while(visited.contains(next))
                {
                        next = waiting.remove();
                }
               
                while (!validURL) 
                {
                		try
                		{
                             doc = Jsoup.connect(next).get();
                             validURL = true;
                        } 
                        catch (IOException e) 
                        {
                              next = waiting.remove();
                        } 
                        catch (IllegalArgumentException e) 
                        {
                              next = waiting.remove();
                        } 
                        catch (NullPointerException e) 
                        {
                              next = waiting.remove();
                        }
                }
                visited.add(next);
                return doc;
        }
}


